% george donnelly - 106338300
% cse 364 - hw 1

trainingDocuments = {};
testingDocuments = {};
tokens = {};
theTestingFiles = {};
theTrainingFiles = {};

%--------- get the file documents to memory
'Getting File Data and Tokens'
fid = fopen('emails/spamtesting.txt');
while ~feof(fid),
    line = fgetl(fid);
    testingDocuments{end+1} = line;
end
fclose(fid);

fid = fopen('emails/hamtesting.txt');
while ~feof(fid),
    line = fgetl(fid);
    testingDocuments{end+1} = line;
   
end
fclose(fid);

fid = fopen('emails/spamtraining.txt');
while ~feof(fid),
    line = fgetl(fid);
    trainingDocuments{end+1} = line;
end
fclose(fid);

fid = fopen('emails/hamtraining.txt');
while ~feof(fid),
    line = fgetl(fid);
    trainingDocuments{end+1} = line;
end
fclose(fid);

%--------- get the tokens to memory
for i=1:length(testingDocuments)
    url = strcat('emails/',testingDocuments{i});
    fid = fopen(url);
    while ~feof(fid),
        line = fgetl(fid);
        token1 = strtok(line);
        tokens{end+1} = token1;
    end
  
    fclose(fid);
end


for i=1:length(trainingDocuments)
    url = strcat('emails/',trainingDocuments{i});
    fid = fopen(url);
    while ~feof(fid),
        line = fgetl(fid);
        token = strtok(line);
        tokens{end+1} = token;
    end
  
    fclose(fid);
end

%--------- get file data to memory
for i=1:length(trainingDocuments)
    url = strcat('emails/',trainingDocuments{i});
    fid = fopen(url);
    theTrainingFiles{end+1} = {textscan(fid, '%s')};
    fclose(fid);
end

for i=1:length(testingDocuments)
    url = strcat('emails/',testingDocuments{i});
    fid = fopen(url);
    theTestingFiles{end+1} = {textscan(fid, '%s')};
    fclose(fid);
end


%---------%get the tokens that appear more than k times and add them to lexicon
lexicon = {};
k = 1;
'Calculating Lexicon...'
for i=1:length(tokens),
    token1 = tokens{i};
    counter = 0;
    for j=1:length(tokens),
        if strcmp(token1,tokens{j}),
            counter = counter + 1;
        end
    end
    
    % if counter is k and token is not a member of lexicon then add it!
    if counter == k,
        if(find(ismember(lexicon, token1)==1)),
            'dont add to lexicon';
        else 
            lexicon{end+1} = token1;
        end
    end
end
'Done Calculating Lexicon'
%print out remove colon
lexicon


%create the training matrix
lsize = length(lexicon);
dsize = length(trainingDocuments);
M = zeros(lsize,dsize,'uint32');

%---------calculate training data
'Calculate Training Data'
for i=1:length(lexicon),
    lexiconToken = lexicon{i};
    
    for k=1:length(theTrainingFiles),
        for j=1:length(theTrainingFiles{k}{1}{1}),
            docToken = theTrainingFiles{k}{1}{1}{j};
            if(strcmp(lexiconToken,docToken)),
                value = M(i,k);
                M(i,k) = value + 1;
            end
        end
    end
    
end

'Done Calculating Training Data'

% print out matrix remove colon
M;

% label vector with spam or ham
l = zeros(1,dsize);
l(1:100) = l(1:100) + 1;

%--------- calculate testing data and classify them

% testing documents label vector 1 for spam 0 for ham
 tl = zeros(1,length(testingDocuments));



'Calculate Testing Data'
    % array where ssds will be stores
    
    ham = 0;
    spam = 0;
    
    for i=1:length(theTestingFiles),
        
        
        tp = [];
        
        % array for word vector of testing documents
        y = zeros(lsize,1);

        for j=1:length(theTestingFiles{i}{1}{1}),
            token = theTestingFiles{i}{1}{1}{j};
            x = find(ismember(lexicon,token)==1);
            y(x) = y(x) + 1;
        end

        % vector sum of y
        sumOfY = sum(y);

        % a vector of sum of colums in M
        vectorOfSums = sum(M);
        
        %compare each testing vector to column vector in M
        for j=1:length(vectorOfSums)
            ssd = (sumOfY - vectorOfSums(j))^2;
            tp(end+1) = ssd;
        end
        
        tp;
        
        % get the smallest value (SSD) and check wither it is spam/ham
        thevalue = min(tp);
        theLexIndex = find(ismember(tp,thevalue)==1);
        if theLexIndex > 100,
            ham = ham +1;
        else
            spam = spam + 1;
        end
       
    end
'Done Calculating Testing Data'

%measure the performence
'Measure Performence'

spam
ham
thespam = spam/100
theham = ham/100

